import eia_backtestingimport pandas as pdimport plotly.graph_objects as goimport plotly.express as pximport osimport datetimeimport requestsimport json
Support for Torch based models not available. To enable them, install "darts", "u8darts[torch]" or "u8darts[all]" (with pip); or "u8darts-torch" or "u8darts-all" (with conda).
/opt/ai_dev_workshop/lib/python3.10/site-packages/statsforecast/core.py:27: TqdmExperimentalWarning:
Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)
series = pd.DataFrame(meta_json["series"])backtesting_output = {}for index, row in series.iterrows():print("Start backtesting for "+ row['subba_id']) df =None df =input[input["subba"] == row['subba_id']] df = df.sort_values(by = ["period"]) par = paramsfor i inrange(len(par)): par[i]["subba"] = row['subba_id'] backtesting_output[row['subba_id']] = eia_backtesting.backtesting(input= df, partitions=p, overlap = overlap, h = h, params = par, experiment_name = experiment_name +"_"+ row['subba_id'], mlflow_path = mlflow_path, overwrite =True, tags = tags)
Code
leaderboard =Nonebest =Nonefor index, row in series.iterrows(): subba = row["subba_id"]print(subba)print(backtesting_output[subba].leaderboard) leader_subba = backtesting_output[subba].leaderboard leader_subba["subba"] = subbaif leaderboard isNone: leaderboard = backtesting_output[subba].leaderboard best = leader_subba[leader_subba["mape"] == leader_subba["mape"].min()]else: leaderboard_temp = backtesting_output[subba].leaderboard leaderboard = leaderboard._append(leaderboard_temp) best = best._append(leader_subba[leader_subba["mape"] == leader_subba["mape"].min()])#best = best[["subba", "model","model_label" ,"mape", "rmse"]] best.to_csv(leaderboard_path, index =False) best
PGAE
forecast_label model_label model mape rmse \
0 2024-06-14 model1 LinearRegressionModel 0.089334 1365.539586
1 2024-06-14 model2 LinearRegressionModel 0.090086 1374.423744
2 2024-06-14 model3 LinearRegressionModel 0.096589 1456.745612
3 2024-06-14 model4 LinearRegressionModel 0.091571 1415.877250
4 2024-06-14 model5 LinearRegressionModel 0.092669 1404.666182
5 2024-06-14 model7 XGBModel 0.091234 1333.922406
coverage comments
0 0.725000 LM model with lags, training with 2 years of h...
1 0.729167 LM model with lags, training with 3 years of h...
2 0.537500 Model 2 with lag 1
3 0.725000 Model 1 with additional lags
4 0.725000 Model 1 with additional lags
5 0.670833 XGBoost with lags
SCE
forecast_label model_label model mape rmse \
0 2024-06-14 model1 LinearRegressionModel 0.052259 824.558770
1 2024-06-14 model2 LinearRegressionModel 0.051878 817.262735
2 2024-06-14 model3 LinearRegressionModel 0.077653 1139.373683
3 2024-06-14 model4 LinearRegressionModel 0.068839 1073.844087
4 2024-06-14 model5 LinearRegressionModel 0.071172 1121.572839
5 2024-06-14 model7 XGBModel 0.077308 1153.257120
coverage comments
0 0.929167 LM model with lags, training with 2 years of h...
1 0.929167 LM model with lags, training with 3 years of h...
2 0.695833 Model 2 with lag 1
3 0.887500 Model 1 with additional lags
4 0.862500 Model 1 with additional lags
5 0.775000 XGBoost with lags
SDGE
forecast_label model_label model mape rmse \
0 2024-06-14 model1 LinearRegressionModel 0.083502 187.714005
1 2024-06-14 model2 LinearRegressionModel 0.083141 188.657150
2 2024-06-14 model3 LinearRegressionModel 0.131489 266.710639
3 2024-06-14 model4 LinearRegressionModel 0.125430 257.151173
4 2024-06-14 model5 LinearRegressionModel 0.123442 251.715517
5 2024-06-14 model7 XGBModel 0.129947 261.224281
coverage comments
0 0.933333 LM model with lags, training with 2 years of h...
1 0.933333 LM model with lags, training with 3 years of h...
2 0.770833 Model 2 with lag 1
3 0.879167 Model 1 with additional lags
4 0.887500 Model 1 with additional lags
5 0.875000 XGBoost with lags
VEA
forecast_label model_label model mape rmse \
0 2024-06-14 model1 LinearRegressionModel 0.100615 18.513913
1 2024-06-14 model2 LinearRegressionModel 0.100654 18.651437
2 2024-06-14 model3 LinearRegressionModel 0.150168 30.266325
3 2024-06-14 model4 LinearRegressionModel 0.127715 25.427010
4 2024-06-14 model5 LinearRegressionModel 0.129275 25.454507
5 2024-06-14 model7 XGBModel 0.114773 22.695727
coverage comments
0 0.887500 LM model with lags, training with 2 years of h...
1 0.833333 LM model with lags, training with 3 years of h...
2 0.787500 Model 2 with lag 1
3 0.850000 Model 1 with additional lags
4 0.850000 Model 1 with additional lags
5 0.858333 XGBoost with lags
forecast_label
model_label
model
mape
rmse
coverage
comments
subba
0
2024-06-14
model1
LinearRegressionModel
0.089334
1365.539586
0.725000
LM model with lags, training with 2 years of h...
PGAE
1
2024-06-14
model2
LinearRegressionModel
0.051878
817.262735
0.929167
LM model with lags, training with 3 years of h...
SCE
1
2024-06-14
model2
LinearRegressionModel
0.083141
188.657150
0.933333
LM model with lags, training with 3 years of h...
SDGE
0
2024-06-14
model1
LinearRegressionModel
0.100615
18.513913
0.887500
LM model with lags, training with 2 years of h...
VEA
Code
print(leaderboard)
forecast_label model_label model mape rmse \
0 2024-06-14 model1 LinearRegressionModel 0.089334 1365.539586
1 2024-06-14 model2 LinearRegressionModel 0.090086 1374.423744
2 2024-06-14 model3 LinearRegressionModel 0.096589 1456.745612
3 2024-06-14 model4 LinearRegressionModel 0.091571 1415.877250
4 2024-06-14 model5 LinearRegressionModel 0.092669 1404.666182
5 2024-06-14 model7 XGBModel 0.091234 1333.922406
0 2024-06-14 model1 LinearRegressionModel 0.052259 824.558770
1 2024-06-14 model2 LinearRegressionModel 0.051878 817.262735
2 2024-06-14 model3 LinearRegressionModel 0.077653 1139.373683
3 2024-06-14 model4 LinearRegressionModel 0.068839 1073.844087
4 2024-06-14 model5 LinearRegressionModel 0.071172 1121.572839
5 2024-06-14 model7 XGBModel 0.077308 1153.257120
0 2024-06-14 model1 LinearRegressionModel 0.083502 187.714005
1 2024-06-14 model2 LinearRegressionModel 0.083141 188.657150
2 2024-06-14 model3 LinearRegressionModel 0.131489 266.710639
3 2024-06-14 model4 LinearRegressionModel 0.125430 257.151173
4 2024-06-14 model5 LinearRegressionModel 0.123442 251.715517
5 2024-06-14 model7 XGBModel 0.129947 261.224281
0 2024-06-14 model1 LinearRegressionModel 0.100615 18.513913
1 2024-06-14 model2 LinearRegressionModel 0.100654 18.651437
2 2024-06-14 model3 LinearRegressionModel 0.150168 30.266325
3 2024-06-14 model4 LinearRegressionModel 0.127715 25.427010
4 2024-06-14 model5 LinearRegressionModel 0.129275 25.454507
5 2024-06-14 model7 XGBModel 0.114773 22.695727
coverage comments subba
0 0.725000 LM model with lags, training with 2 years of h... PGAE
1 0.729167 LM model with lags, training with 3 years of h... PGAE
2 0.537500 Model 2 with lag 1 PGAE
3 0.725000 Model 1 with additional lags PGAE
4 0.725000 Model 1 with additional lags PGAE
5 0.670833 XGBoost with lags PGAE
0 0.929167 LM model with lags, training with 2 years of h... SCE
1 0.929167 LM model with lags, training with 3 years of h... SCE
2 0.695833 Model 2 with lag 1 SCE
3 0.887500 Model 1 with additional lags SCE
4 0.862500 Model 1 with additional lags SCE
5 0.775000 XGBoost with lags SCE
0 0.933333 LM model with lags, training with 2 years of h... SDGE
1 0.933333 LM model with lags, training with 3 years of h... SDGE
2 0.770833 Model 2 with lag 1 SDGE
3 0.879167 Model 1 with additional lags SDGE
4 0.887500 Model 1 with additional lags SDGE
5 0.875000 XGBoost with lags SDGE
0 0.887500 LM model with lags, training with 2 years of h... VEA
1 0.833333 LM model with lags, training with 3 years of h... VEA
2 0.787500 Model 2 with lag 1 VEA
3 0.850000 Model 1 with additional lags VEA
4 0.850000 Model 1 with additional lags VEA
5 0.858333 XGBoost with lags VEA